<!DOCTYPE html>


<html lang="zh-CN">


<head>
  <meta charset="utf-8" />
    
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" />
  <title>
    K8S使用ceph-csi持久化存储之CephFS.md |  
  </title>
  <meta name="generator" content="hexo-theme-ayer">
  
  <link rel="shortcut icon" href="/favicon.ico" />
  
  
<link rel="stylesheet" href="/dist/main.css">

  
<link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/Shen-Yu/cdn/css/remixicon.min.css">

  
<link rel="stylesheet" href="/css/custom.css">

  
  
<script src="https://cdn.jsdelivr.net/npm/pace-js@1.0.2/pace.min.js"></script>

  
  

  

</head>

</html>

<body>
  <div id="app">
    
      
    <main class="content on">
      <section class="outer">
  <article
  id="post-k8s/K8S使用ceph-csi持久化存储之CephFS"
  class="article article-type-post"
  itemscope
  itemprop="blogPost"
  data-scroll-reveal
>
  <div class="article-inner">
    
    <header class="article-header">
       
<h1 class="article-title sea-center" style="border-left:0" itemprop="name">
  K8S使用ceph-csi持久化存储之CephFS.md
</h1>
 

    </header>
     
    <div class="article-meta">
      <a href="/2020/11/11/k8s/K8S%E4%BD%BF%E7%94%A8ceph-csi%E6%8C%81%E4%B9%85%E5%8C%96%E5%AD%98%E5%82%A8%E4%B9%8BCephFS/" class="article-date">
  <time datetime="2020-11-10T16:00:00.000Z" itemprop="datePublished">2020-11-11</time>
</a> 
  <div class="article-category">
    <a class="article-category-link" href="/categories/k8s/">k8s</a>
  </div>
  
<div class="word_count">
    <span class="post-time">
        <span class="post-meta-item-icon">
            <i class="ri-quill-pen-line"></i>
            <span class="post-meta-item-text"> 字数统计:</span>
            <span class="post-count">2.8k</span>
        </span>
    </span>

    <span class="post-time">
        &nbsp; | &nbsp;
        <span class="post-meta-item-icon">
            <i class="ri-book-open-line"></i>
            <span class="post-meta-item-text"> 阅读时长≈</span>
            <span class="post-count">15 分钟</span>
        </span>
    </span>
</div>
 
    </div>
      
    <div class="tocbot"></div>




  
    <div class="article-entry" itemprop="articleBody">
       
  <h1 id="K8S使用ceph-csi持久化存储之CephFS"><a href="#K8S使用ceph-csi持久化存储之CephFS" class="headerlink" title="K8S使用ceph-csi持久化存储之CephFS"></a>K8S使用ceph-csi持久化存储之CephFS</h1><p>继上一篇介绍Ceph-CSI的RBD对接K8S的文章（<a target="_blank" rel="noopener" href="http://mp.weixin.qq.com/s?__biz=MzI5ODQ2MzI3NQ==&mid=2247493191&idx=2&sn=f11084b671288655ec87154deef177b2&chksm=eca7df03dbd0561583efeaef2b9df1e11012f29cd7de681c507b4df029e03943ce97f95810d9&scene=21#wechat_redirect">K8S使用ceph-csi持久化存储之RBD</a>）后，本篇章介绍了Ceph-CSI的CephFS对接K8S的使用。请食用！*</p>
<h3 id="一、集群和组件版本"><a href="#一、集群和组件版本" class="headerlink" title="一、集群和组件版本"></a>一、集群和组件版本</h3><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">K8S集群：1.17.3+</span><br><span class="line">Ceph集群：Octopus（stables）</span><br><span class="line">Ceph-CSI：release-v3.1</span><br><span class="line">snapshotter-controller：release-2.1</span><br><span class="line">Linue kernel：3.10.0-1127.19.1.el7.x86_64 +</span><br><span class="line">注意：1）CephFS的快照功能需要Ceph版本为O版；2）创建了快照的pvc挂载需要内核4.14版本。</span><br></pre></td></tr></table></figure>

<ul>
<li>镜像版本：</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">docker pull quay.io/k8scsi/csi-snapshotter:v2.1.1</span><br><span class="line">docker pull quay.io/k8scsi/csi-snapshotter:v2.1.0</span><br><span class="line">docker pull quay.io/k8scsi/csi-resizer:v0.5.0</span><br><span class="line">docker pull quay.io/k8scsi/csi-provisioner:v1.6.0</span><br><span class="line">docker pull quay.io/k8scsi/csi-node-driver-registrar:v1.3.0</span><br><span class="line">docker pull quay.io/k8scsi/csi-attacher:v2.1.1</span><br><span class="line">docker pull quay.io/cephcsi/cephcsi:v3.1-canary</span><br><span class="line">docker pull quay.io/k8scsi/snapshot-controller:v2.0.1</span><br></pre></td></tr></table></figure>

<h3 id="二、部署"><a href="#二、部署" class="headerlink" title="二、部署"></a>二、部署</h3><h4 id="1）部署Ceph-CSI"><a href="#1）部署Ceph-CSI" class="headerlink" title="1）部署Ceph-CSI"></a>1）部署Ceph-CSI</h4><h5 id="1-1）克隆代码"><a href="#1-1）克隆代码" class="headerlink" title="1.1）克隆代码"></a>1.1）克隆代码</h5><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> git <span class="built_in">clone</span> https://github.com/ceph/ceph-csi.git</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> <span class="built_in">cd</span> ceph-csi/deploy/cephfs/kubernetes</span></span><br></pre></td></tr></table></figure>

<h5 id="1-2）修改yaml文件"><a href="#1-2）修改yaml文件" class="headerlink" title="1.2）修改yaml文件"></a>1.2）修改yaml文件</h5><p><em>1.2.1）配置csi-config-map.yaml文件链接ceph集群的信息</em></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> cat csi-config-map.yaml</span></span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: ConfigMap</span><br><span class="line">data:</span><br><span class="line">  config.json: |-</span><br><span class="line">    [</span><br><span class="line">      &#123;</span><br><span class="line">        &quot;clusterID&quot;: &quot;c7b4xxf7-c61e-4668-9xx0-82c9xx5e3696&quot;,    // 通过ceph集群的ID</span><br><span class="line">        &quot;monitors&quot;: [</span><br><span class="line">          &quot;xxx.xxx.xxx.xxx:6789&quot;</span><br><span class="line">        ]</span><br><span class="line">      &#125;</span><br><span class="line">    ]</span><br><span class="line">metadata:</span><br><span class="line">  name: ceph-csi-config</span><br></pre></td></tr></table></figure>

<p><em>1.2.2）部署cephfs相关的CSI</em></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f ceph-csi/deploy/cephfs/kubernetes/</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl get pods | grep cephfs</span></span><br><span class="line">csi-cephfsplugin-8mvmf                          3/3     Running   0          2d21h</span><br><span class="line">csi-cephfsplugin-provisioner-78f64778db-42xwc   6/6     Running   0          2d21h</span><br><span class="line">csi-cephfsplugin-provisioner-78f64778db-jzpvd   6/6     Running   0          2d21h</span><br><span class="line">csi-cephfsplugin-provisioner-78f64778db-ntcmd   6/6     Running   0          2d21h</span><br><span class="line">csi-cephfsplugin-zlqzg                          3/3     Running   0          2d21h</span><br></pre></td></tr></table></figure>

<h4 id="2-快照功能需要安装快照控制器支持："><a href="#2-快照功能需要安装快照控制器支持：" class="headerlink" title="2)快照功能需要安装快照控制器支持："></a>2)快照功能需要安装快照控制器支持：</h4><p><em>由于上篇 “K8S使用ceph-csi持久化存储之RBD” 以及部署，这里不需要重复部署。至此，Ceph-CSI和snapshot-controller安装完成。下面进行功能测试。测试功能前需要在ceph集群中创建对应的存储池：</em></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br></pre></td><td class="code"><pre><span class="line">// 查看集群状态</span><br><span class="line"><span class="meta">#</span><span class="bash"> ceph -s</span></span><br><span class="line">  cluster:</span><br><span class="line">    id:     c7b43ef7-c61e-4668-9970-82c9775e3696</span><br><span class="line">    health: HEALTH_OK</span><br><span class="line"> </span><br><span class="line">  services:</span><br><span class="line">    mon: 1 daemons, quorum cka-node-01 (age 24h)</span><br><span class="line">    mgr: cka-node-01(active, since 24h), standbys: cka-node-02, cka-node-03</span><br><span class="line">    mds: cephfs:1 &#123;0=cka-node-01=up:active&#125; 2 up:standby</span><br><span class="line">    osd: 3 osds: 3 up, 3 in</span><br><span class="line">    rgw: 1 daemon active (cka-node-01)</span><br><span class="line"> </span><br><span class="line">  task status:</span><br><span class="line">    scrub status:</span><br><span class="line">        mds.cka-node-01: idle</span><br><span class="line"> </span><br><span class="line">  data:</span><br><span class="line">    pools:   7 pools, 184 pgs</span><br><span class="line">    objects: 827 objects, 1.7 GiB</span><br><span class="line">    usage:   8.1 GiB used, 52 GiB / 60 GiB avail</span><br><span class="line">    pgs:     184 active+clean</span><br><span class="line"> </span><br><span class="line">  io:</span><br><span class="line">    client:   32 KiB/s rd, 0 B/s wr, 31 op/s rd, 21 op/s wr</span><br><span class="line"> </span><br><span class="line">// 创建cephfs存储池fs_metadata，fs_data</span><br><span class="line"><span class="meta">#</span><span class="bash"> ceph osd pool create fs_metadata 8 8</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> ceph osd pool create fs_data 8 8</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> ceph fs new cephfs fs_metadata fs_data</span></span><br><span class="line"></span><br><span class="line"> </span><br><span class="line">// 获取集群信息和查看用户key</span><br><span class="line"><span class="meta">#</span><span class="bash"> ceph mon dump</span></span><br><span class="line">dumped monmap epoch 3</span><br><span class="line">epoch 3</span><br><span class="line">fsid c7b43ef7-c61e-4668-9970-82c9775e3696</span><br><span class="line">last_changed 2020-09-11 11:05:25.529648</span><br><span class="line">created 2020-09-10 16:22:52.967856</span><br><span class="line">min_mon_release 14 (nautilus)</span><br><span class="line">0: [v2:10.0.xxx.xxx0:3300/0,v1:10.0.xxx.xxx:6789/0] mon.cka-node-01</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> ceph auth get client.admin</span></span><br><span class="line">exported keyring for client.admin</span><br><span class="line">[client.admin]</span><br><span class="line">	key = AQBg4llf+9CAGxdsx4tQzS+0OssssiTEQ==</span><br><span class="line">	caps mds = &quot;allow *&quot;</span><br><span class="line">	caps mgr = &quot;allow *&quot;</span><br><span class="line">	caps mon = &quot;allow *&quot;</span><br><span class="line">	caps osd = &quot;allow *&quot;</span><br></pre></td></tr></table></figure>

<h3 id="三、验证"><a href="#三、验证" class="headerlink" title="三、验证"></a>三、验证</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">验证如下功能：</span><br><span class="line">1）创建cephfs类型pvc给pod使用；</span><br><span class="line">2）创建cephfs类型pvc的快照，并验证基于快照恢复的可用性（CephFS的快照功能需要Ceph版本为O版）；</span><br><span class="line">3）创建快照后的pvc重复创建挂载（创建了快照的pvc挂载需要内核4.14版本）；</span><br></pre></td></tr></table></figure>

<h5 id="1、创建cephfs类型pvc给pod使用："><a href="#1、创建cephfs类型pvc给pod使用：" class="headerlink" title="1、创建cephfs类型pvc给pod使用："></a>1、创建cephfs类型pvc给pod使用：</h5><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br></pre></td><td class="code"><pre><span class="line">1.1) 创建连接ceph集群的秘钥</span><br><span class="line"><span class="meta">#</span><span class="bash"> cat secret.yaml</span></span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: Secret</span><br><span class="line">metadata:</span><br><span class="line">  name: csi-cephfs-secret</span><br><span class="line">  namespace: default</span><br><span class="line">stringData:</span><br><span class="line"><span class="meta">  #</span><span class="bash"> Required <span class="keyword">for</span> statically provisioned volumes</span></span><br><span class="line">  userID: admin</span><br><span class="line">  userKey: AQBg4llf+9CAGdsAds4tQzS+0O7dscB5ZTiTEQ==</span><br><span class="line"></span><br><span class="line"><span class="meta">  #</span><span class="bash"> Required <span class="keyword">for</span> dynamically provisioned volumes</span></span><br><span class="line">  adminID: admin</span><br><span class="line">  adminKey: AQBg4llf+9CAGdsAds4tQzS+0O7dscB5ZTiTEQ==</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f secret.yaml</span></span><br><span class="line">1.2) 创建storeclass</span><br><span class="line"><span class="meta">#</span><span class="bash"> cat storageclass.yaml</span></span><br><span class="line">---</span><br><span class="line">apiVersion: storage.k8s.io/v1</span><br><span class="line">kind: StorageClass</span><br><span class="line">metadata:</span><br><span class="line">  name: csi-cephfs-sc</span><br><span class="line">provisioner: cephfs.csi.ceph.com</span><br><span class="line">parameters:</span><br><span class="line">  clusterID: c7b43ef7-c61e-4668-9970-82c9775e3696</span><br><span class="line">  fsName: cephfs</span><br><span class="line">  pool: fs_data</span><br><span class="line">  rootPath: /test</span><br><span class="line">  csi.storage.k8s.io/provisioner-secret-name: csi-cephfs-secret</span><br><span class="line">  csi.storage.k8s.io/provisioner-secret-namespace: default</span><br><span class="line">  csi.storage.k8s.io/controller-expand-secret-name: csi-cephfs-secret</span><br><span class="line">  csi.storage.k8s.io/controller-expand-secret-namespace: default</span><br><span class="line">  csi.storage.k8s.io/node-stage-secret-name: csi-cephfs-secret</span><br><span class="line">  csi.storage.k8s.io/node-stage-secret-namespace: default</span><br><span class="line">reclaimPolicy: Delete</span><br><span class="line">allowVolumeExpansion: true</span><br><span class="line">mountOptions:</span><br><span class="line">  - discard</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f storageclass.yaml</span></span><br><span class="line">1.3)基于storeclass创建pvc</span><br><span class="line"><span class="meta">#</span><span class="bash"> cat pvc.yaml</span></span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: PersistentVolumeClaim</span><br><span class="line">metadata:</span><br><span class="line">  name: csi-cephfs-pvc</span><br><span class="line">spec:</span><br><span class="line">  accessModes:</span><br><span class="line">    - ReadWriteMany</span><br><span class="line">  resources:</span><br><span class="line">    requests:</span><br><span class="line">      storage: 1Gi</span><br><span class="line">  storageClassName: csi-cephfs-sc</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f pvc.yaml</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl get pvc csi-cephfs-pvc</span></span><br><span class="line">NAME             STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS    AGE</span><br><span class="line">csi-cephfs-pvc   Bound    pvc-72b5c1e0-b2d5-4f93-8779-96cb03efa733   1Gi        RWX            csi-cephfs-sc   2d21h</span><br><span class="line">1.4）创建pod应用pvc</span><br><span class="line"><span class="meta">#</span><span class="bash"> cat pod.yaml</span></span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: Pod</span><br><span class="line">metadata:</span><br><span class="line">  name: csi-cephfs-demo-pod</span><br><span class="line">spec:</span><br><span class="line">  containers:</span><br><span class="line">    - name: web-server</span><br><span class="line">      image: nginx</span><br><span class="line">      volumeMounts:</span><br><span class="line">        - name: mypvc</span><br><span class="line">          mountPath: /var/lib/www</span><br><span class="line">  volumes:</span><br><span class="line">    - name: mypvc</span><br><span class="line">      persistentVolumeClaim:</span><br><span class="line">        claimName: csi-cephfs-pvc</span><br><span class="line">        readOnly: false</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f pod.yaml</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl get pods csi-cephfs-demo-pod</span></span><br><span class="line">NAME                  READY   STATUS    RESTARTS   AGE</span><br><span class="line">csi-cephfs-demo-pod   1/1     Running   0          2d21h</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl <span class="built_in">exec</span> -ti csi-cephfs-demo-pod -- bash</span></span><br><span class="line">root@csi-cephfs-demo-pod:/# df -h</span><br><span class="line">Filesystem               Size  Used Avail Use% Mounted on</span><br><span class="line">overlay                  199G  7.9G  191G   4% /</span><br><span class="line">tmpfs                     64M     0   64M   0% /dev</span><br><span class="line">tmpfs                    7.9G     0  7.9G   0% /sys/fs/cgroup</span><br><span class="line">/dev/mapper/centos-root  199G  7.9G  191G   4% /etc/hosts</span><br><span class="line">shm                       64M     0   64M   0% /dev/shm</span><br><span class="line">ceph-fuse                1.0G     0  1.0G   0% /var/lib/www</span><br><span class="line">tmpfs                    7.9G   12K  7.9G   1% /run/secrets/kubernetes.io/serviceaccount</span><br><span class="line">tmpfs                    7.9G     0  7.9G   0% /proc/acpi</span><br><span class="line">tmpfs                    7.9G     0  7.9G   0% /proc/scsi</span><br><span class="line">tmpfs                    7.9G     0  7.9G   0% /sys/firmware</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash"> 写入文件，用于后续快照验证</span></span><br><span class="line">root@csi-cephfs-demo-pod:/# cd /var/lib/www;echo &quot;abce&quot; &gt; test</span><br><span class="line">root@csi-cephfs-demo-pod:/var/lib/www# cat test</span><br><span class="line">abce</span><br></pre></td></tr></table></figure>

<h5 id="2）创建cephfs类型pvc的快照，并验证基于快照恢复的可用性（CephFS的快照功能需要Ceph版本为O版）："><a href="#2）创建cephfs类型pvc的快照，并验证基于快照恢复的可用性（CephFS的快照功能需要Ceph版本为O版）：" class="headerlink" title="2）创建cephfs类型pvc的快照，并验证基于快照恢复的可用性（CephFS的快照功能需要Ceph版本为O版）："></a>2）创建cephfs类型pvc的快照，并验证基于快照恢复的可用性（CephFS的快照功能需要Ceph版本为O版）：</h5><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br></pre></td><td class="code"><pre><span class="line">2.1)创建上一步pvc的快照</span><br><span class="line"><span class="meta">#</span><span class="bash"> cat snapshot.yaml</span></span><br><span class="line">---</span><br><span class="line">apiVersion: snapshot.storage.k8s.io/v1beta1</span><br><span class="line">kind: VolumeSnapshot</span><br><span class="line">metadata:</span><br><span class="line">  name: cephfs-pvc-snapshot</span><br><span class="line">spec:</span><br><span class="line">  volumeSnapshotClassName: csi-cephfsplugin-snapclass</span><br><span class="line">  source:</span><br><span class="line">    persistentVolumeClaimName: csi-cephfs-pvc</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f snapshot.yaml</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl get VolumeSnapshot cephfs-pvc-snapshot</span></span><br><span class="line">NAME                  READYTOUSE   SOURCEPVC        SOURCESNAPSHOTCONTENT   RESTORESIZE   SNAPSHOTCLASS                SNAPSHOTCONTENT                                    CREATIONTIME   AGE</span><br><span class="line">cephfs-pvc-snapshot   true         csi-cephfs-pvc                           1Gi           csi-cephfsplugin-snapclass   snapcontent-6100977b-96e0-43ae-903a-e8136b0f6ced   2d13h          2d21h</span><br><span class="line">2.2)创建基于快照恢复的pvc</span><br><span class="line"><span class="meta">#</span><span class="bash"> cat pvc-restore.yaml</span></span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: PersistentVolumeClaim</span><br><span class="line">metadata:</span><br><span class="line">  name: cephfs-pvc-restore</span><br><span class="line">spec:</span><br><span class="line">  storageClassName: csi-cephfs-sc</span><br><span class="line">  dataSource:</span><br><span class="line">    name: cephfs-pvc-snapshot</span><br><span class="line">    kind: VolumeSnapshot</span><br><span class="line">    apiGroup: snapshot.storage.k8s.io</span><br><span class="line">  accessModes:</span><br><span class="line">    - ReadWriteMany</span><br><span class="line">  resources:</span><br><span class="line">    requests:</span><br><span class="line">      storage: 1Gi</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f pvc-restore.yaml</span></span><br><span class="line">2.3)创建pod应用快照恢复的pvc</span><br><span class="line"><span class="meta">#</span><span class="bash"> cat pod-restore.yaml</span></span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: Pod</span><br><span class="line">metadata:</span><br><span class="line">  name: csi-cephfs-restore-demo-pod</span><br><span class="line">spec:</span><br><span class="line">  containers:</span><br><span class="line">    - name: web-server</span><br><span class="line">      image: nginx</span><br><span class="line">      volumeMounts:</span><br><span class="line">        - name: mypvc</span><br><span class="line">          mountPath: /var/lib/www/html</span><br><span class="line">  volumes:</span><br><span class="line">    - name: mypvc</span><br><span class="line">      persistentVolumeClaim:</span><br><span class="line">        claimName: cephfs-pvc-restore</span><br><span class="line">        readOnly: false</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f pod-restore.yaml</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl get pods csi-cephfs-restore-demo-pod</span></span><br><span class="line">NAME                          READY   STATUS    RESTARTS   AGE</span><br><span class="line">csi-cephfs-restore-demo-pod   1/1     Running   0          2d21h</span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl <span class="built_in">exec</span> -ti csi-cephfs-restore-demo-pod -- bash</span></span><br><span class="line">root@csi-cephfs-restore-demo-pod:/# df -h</span><br><span class="line">Filesystem               Size  Used Avail Use% Mounted on</span><br><span class="line">overlay                  199G  7.9G  191G   4% /</span><br><span class="line">tmpfs                     64M     0   64M   0% /dev</span><br><span class="line">tmpfs                    7.9G     0  7.9G   0% /sys/fs/cgroup</span><br><span class="line">/dev/mapper/centos-root  199G  7.9G  191G   4% /etc/hosts</span><br><span class="line">shm                       64M     0   64M   0% /dev/shm</span><br><span class="line">ceph-fuse                1.0G     0  1.0G   0% /var/lib/www/html</span><br><span class="line">tmpfs                    7.9G   12K  7.9G   1% /run/secrets/kubernetes.io/serviceaccount</span><br><span class="line">tmpfs                    7.9G     0  7.9G   0% /proc/acpi</span><br><span class="line">tmpfs                    7.9G     0  7.9G   0% /proc/scsi</span><br><span class="line">tmpfs                    7.9G     0  7.9G   0% /sys/firmware</span><br><span class="line"></span><br><span class="line">root@csi-cephfs-restore-demo-pod:/# cd /var/lib/www/html</span><br><span class="line">root@csi-cephfs-restore-demo-pod:/var/lib/www/html# cat test</span><br><span class="line">abce</span><br><span class="line"> </span><br><span class="line">//基于快照恢复数据功能正常</span><br></pre></td></tr></table></figure>

<h5 id="3）创建快照后的pvc重复创建挂载（创建了快照的pvc挂载需要内核4-14版本，RBD类型也需要升级）："><a href="#3）创建快照后的pvc重复创建挂载（创建了快照的pvc挂载需要内核4-14版本，RBD类型也需要升级）：" class="headerlink" title="3）创建快照后的pvc重复创建挂载（创建了快照的pvc挂载需要内核4.14版本，RBD类型也需要升级）："></a>3）创建快照后的pvc重复创建挂载（创建了快照的pvc挂载需要内核4.14版本，RBD类型也需要升级）：</h5><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre></td><td class="code"><pre><span class="line">3.1)删除csi-cephfs-demo-pod这个pod后重建</span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl delete -f pod.yaml</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f pod.yaml</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl get pods csi-cephfs-demo-pod</span></span><br><span class="line">NAME                  READY   STATUS    RESTARTS   AGE</span><br><span class="line">csi-cephfs-demo-pod   0/1     ContainerCreating   0          2d22h</span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl describe pods csi-cephfs-demo-pod</span></span><br><span class="line">...</span><br><span class="line">Events:</span><br><span class="line">  Type     Reason                  Age              From                     Message</span><br><span class="line">  ----     ------                  ----             ----                     -------</span><br><span class="line">  Normal   Scheduled               &lt;unknown&gt;        default-scheduler        Successfully assigned jonathan/csi-rbd-demo-pod to 10.21.xxx.xxx</span><br><span class="line">  Normal   SuccessfulAttachVolume  25s              attachdetach-controller  AttachVolume.Attach succeeded for volume &quot;pvc-d6ed5203-36c7-454b-8a8e-b49a084a0531&quot;</span><br><span class="line">  Warning  FailedMount             0s (x5 over 9s)  kubelet, 10.21.xxx.xxx    MountVolume.MountDevice failed for volume &quot;pvc-d6ed5203-36c7-454b-8a8e-b49a084a0531&quot; : rpc error: code = Internal desc = rbd: map failed with error an error (exit status 6) occurred while running rbd args: [--id csi -m 10.xxx.xxx.xxx --keyfile=***stripped*** map volumes/csi-vol-9f2dcd03-f97e-11ea-8104-a608005a6960 --device-type krbd], rbd error output: rbd: sysfs write failed</span><br><span class="line">rbd: map failed: (6) No such device or address</span><br><span class="line">3.2）经过分析，该报错是因为node节点的Linux内核版本不支持，需要升级内核。此时的内核版本是3.10版本，升级到4.14版本后问题解决。如下：</span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl get node -o wide</span></span><br><span class="line">NAME          STATUS   ROLES    AGE   VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION            CONTAINER-RUNTIME</span><br><span class="line">cka-node-01   Ready    master   60d   v1.17.3   10.0.xxx.xxx   &lt;none&gt;        CentOS Linux 7 (Core)   4.14.119-200.el7.x86_64   docker://19.3.12</span><br><span class="line">cka-node-02   Ready    &lt;none&gt;   60d   v1.17.3   10.0.xxx.xxx   &lt;none&gt;        CentOS Linux 7 (Core)   4.14.119-200.el7.x86_64   docker://19.3.12</span><br><span class="line">cka-node-03   Ready    &lt;none&gt;   60d   v1.17.3   10.0.xxx.xxx   &lt;none&gt;        CentOS Linux 7 (Core)   4.14.119-200.el7.x86_64   docker://19.3.12</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl get pods csi-cephfs-demo-pod</span></span><br><span class="line">NAME                  READY   STATUS    RESTARTS   AGE</span><br><span class="line">csi-cephfs-demo-pod   1/1     Running   0          2d01h</span><br><span class="line">注意：目前，每个CephFS文件系统限制为400个快照，如果PVC有快照，也无法删除，删除PVC之前，请确保删除PVC上的所有快照。</span><br></pre></td></tr></table></figure>

<h5 id="其他参考"><a href="#其他参考" class="headerlink" title="其他参考"></a>其他参考</h5><h5 id="3）扩容pvc大小："><a href="#3）扩容pvc大小：" class="headerlink" title="3）扩容pvc大小："></a>3）扩容pvc大小：</h5><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br></pre></td><td class="code"><pre><span class="line">3.1)修改rbd-pvc的容量大小</span><br><span class="line"><span class="meta">#</span><span class="bash"> cat pvc.yaml</span></span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: PersistentVolumeClaim</span><br><span class="line">metadata:</span><br><span class="line">  name: rbd-pvc</span><br><span class="line">spec:</span><br><span class="line">  accessModes:</span><br><span class="line">    - ReadWriteOnce</span><br><span class="line">  resources:</span><br><span class="line">    requests:</span><br><span class="line">      storage: 100Gi    // 由1G改为100G</span><br><span class="line">  storageClassName: csi-rbd-sc</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f pvc.yaml</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl get pvc rbd-pvc</span></span><br><span class="line">NAME      STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE</span><br><span class="line">rbd-pvc   Bound    pvc-11b931b0-7cb5-40e1-815b-c15659310593   100Gi      RWO            csi-rbd-sc     40h</span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl <span class="built_in">exec</span> -ti csi-rbd-demo-pod -- bash</span></span><br><span class="line">root@csi-rbd-demo-pod:/# df -h</span><br><span class="line">Filesystem               Size  Used Avail Use% Mounted on</span><br><span class="line">overlay                  199G  7.4G  192G   4% /</span><br><span class="line">tmpfs                     64M     0   64M   0% /dev</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /sys/fs/cgroup</span><br><span class="line">/dev/mapper/centos-root  199G  7.4G  192G   4% /etc/hosts</span><br><span class="line">shm                       64M     0   64M   0% /dev/shm</span><br><span class="line">/dev/rbd0                 99G  6.8M   99G   1% /var/lib/www/html    // 扩容正常</span><br><span class="line">tmpfs                    7.8G   12K  7.8G   1% /run/secrets/kubernetes.io/serviceaccount</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /proc/acpi</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /proc/scsi</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /sys/firmware</span><br><span class="line"> </span><br><span class="line">// 再次写入数据用于后续第二次创建快照</span><br><span class="line">root@csi-rbd-demo-pod:# cd /var/lib/www/html;mkdir test;echo &quot;abc&quot; &gt; test/demo.txt;echo &quot;abc&quot; &gt;&gt; /var/lib/www/html/demo/test.txt</span><br><span class="line">root@csi-rbd-demo-pod:/var/lib/www/html# cat test/demo.txt</span><br><span class="line">abc</span><br><span class="line">root@csi-rbd-demo-pod:/var/lib/www/html# cat demo/test.txt</span><br><span class="line">snapshot test</span><br><span class="line">abc</span><br></pre></td></tr></table></figure>

<h5 id="4）同一个pvc重复创建快照："><a href="#4）同一个pvc重复创建快照：" class="headerlink" title="4）同一个pvc重复创建快照："></a>4）同一个pvc重复创建快照：</h5><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br></pre></td><td class="code"><pre><span class="line">4.1)再次对rbd-pvc创建快照</span><br><span class="line"><span class="meta">#</span><span class="bash"> cat snapshot-1.yaml</span></span><br><span class="line">---</span><br><span class="line">apiVersion: snapshot.storage.k8s.io/v1beta1</span><br><span class="line">kind: VolumeSnapshot</span><br><span class="line">metadata:</span><br><span class="line">  name: rbd-pvc-snapshot-1</span><br><span class="line">spec:</span><br><span class="line">  volumeSnapshotClassName: csi-rbdplugin-snapclass</span><br><span class="line">  source:</span><br><span class="line">    persistentVolumeClaimName: rbd-pvc</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f snapshot-1.yaml</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl get VolumeSnapshot rbd-pvc-snapshot-1</span></span><br><span class="line">NAME                 READYTOUSE   SOURCEPVC   SOURCESNAPSHOTCONTENT   RESTORESIZE   SNAPSHOTCLASS             SNAPSHOTCONTENT                                    CREATIONTIME   AGE</span><br><span class="line">rbd-pvc-snapshot-1   true         rbd-pvc                             100Gi         csi-rbdplugin-snapclass   snapcontent-b82dceb0-7ba6-4a3e-88ab-2220b729d85f   18h            18h</span><br><span class="line">4.2)基于rbd-pvc-snapshot-1快照恢复pvc</span><br><span class="line"><span class="meta">#</span><span class="bash"> cat pvc-restore-1.yaml</span></span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: PersistentVolumeClaim</span><br><span class="line">metadata:</span><br><span class="line">  name: rbd-pvc-restore-1</span><br><span class="line">spec:</span><br><span class="line">  storageClassName: csi-rbd-sc</span><br><span class="line">  dataSource:</span><br><span class="line">    name: rbd-pvc-snapshot-1</span><br><span class="line">    kind: VolumeSnapshot</span><br><span class="line">    apiGroup: snapshot.storage.k8s.io</span><br><span class="line">  accessModes:</span><br><span class="line">    - ReadWriteOnce</span><br><span class="line">  resources:</span><br><span class="line">    requests:</span><br><span class="line">      storage: 100Gi</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f pvc-restore-1.yaml</span></span><br><span class="line">4.3)创建pod引用rbd-pvc-restore-1恢复的pvc</span><br><span class="line"><span class="meta">#</span><span class="bash"> cat pod-restore-1.yaml</span></span><br><span class="line">---</span><br><span class="line">apiVersion: v1</span><br><span class="line">kind: Pod</span><br><span class="line">metadata:</span><br><span class="line">  name: csi-rbd-restore-demo-pod-1</span><br><span class="line">spec:</span><br><span class="line">  containers:</span><br><span class="line">    - name: web-server</span><br><span class="line">      image: nginx</span><br><span class="line">      volumeMounts:</span><br><span class="line">        - name: mypvc</span><br><span class="line">          mountPath: /var/lib/www/html</span><br><span class="line">  volumes:</span><br><span class="line">    - name: mypvc</span><br><span class="line">      persistentVolumeClaim:</span><br><span class="line">        claimName: rbd-pvc-restore-1</span><br><span class="line">        readOnly: false</span><br><span class="line"> </span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl apply -f pod-restore-1.yaml</span></span><br><span class="line">NAME                         READY   STATUS    RESTARTS   AGE</span><br><span class="line">csi-rbd-restore-demo-pod-1   1/1     Running   0          18h</span><br><span class="line"><span class="meta">#</span><span class="bash"> kubectl <span class="built_in">exec</span> -ti csi-rbd-restore-demo-pod-1 -- bash</span></span><br><span class="line">root@csi-rbd-restore-demo-pod-1:/# df -h</span><br><span class="line">Filesystem               Size  Used Avail Use% Mounted on</span><br><span class="line">overlay                  199G  7.4G  192G   4% /</span><br><span class="line">tmpfs                     64M     0   64M   0% /dev</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /sys/fs/cgroup</span><br><span class="line">/dev/mapper/centos-root  199G  7.4G  192G   4% /etc/hosts</span><br><span class="line">shm                       64M     0   64M   0% /dev/shm</span><br><span class="line">/dev/rbd4                 99G  6.8M   99G   1% /var/lib/www/html</span><br><span class="line">tmpfs                    7.8G   12K  7.8G   1% /run/secrets/kubernetes.io/serviceaccount</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /proc/acpi</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /proc/scsi</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /sys/firmware</span><br><span class="line">root@csi-rbd-restore-demo-pod-1:/# cd /var/lib/www/html</span><br><span class="line">root@csi-rbd-restore-demo-pod-1:/var/lib/www/html# cat demo/test.txt</span><br><span class="line">snapshot test</span><br><span class="line">abc</span><br><span class="line">root@csi-rbd-restore-demo-pod-1:/var/lib/www/html# cat test/demo.txt</span><br><span class="line">abc</span><br><span class="line"> </span><br><span class="line">// 至此验证扩容后的pvc，二次创建的快照恢复数据功能正常</span><br><span class="line"> </span><br><span class="line">// 查看第一个创建的快照中是否有后续添加的文件数据,如下数据还是第一个快照创建时数据</span><br><span class="line">[root@cka-node-01 rbd]# kubectl exec -ti csi-rbd-restore-demo-pod -- bash</span><br><span class="line">root@csi-rbd-restore-demo-pod:/# df -h</span><br><span class="line">Filesystem               Size  Used Avail Use% Mounted on</span><br><span class="line">overlay                  199G  7.4G  192G   4% /</span><br><span class="line">tmpfs                     64M     0   64M   0% /dev</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /sys/fs/cgroup</span><br><span class="line">/dev/mapper/centos-root  199G  7.4G  192G   4% /etc/hosts</span><br><span class="line">shm                       64M     0   64M   0% /dev/shm</span><br><span class="line">/dev/rbd3                976M  2.6M  958M   1% /var/lib/www/html</span><br><span class="line">tmpfs                    7.8G   12K  7.8G   1% /run/secrets/kubernetes.io/serviceaccount</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /proc/acpi</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /proc/scsi</span><br><span class="line">tmpfs                    7.8G     0  7.8G   0% /sys/firmware</span><br><span class="line">root@csi-rbd-restore-demo-pod:/# cd /var/lib/www/html</span><br><span class="line">root@csi-rbd-restore-demo-pod:/var/lib/www/html# cat demo/test.txt</span><br><span class="line">snapshot test</span><br><span class="line">root@csi-rbd-restore-demo-pod:/var/lib/www/html# ls</span><br><span class="line">demo  lost+found</span><br></pre></td></tr></table></figure>

<p><em>完</em></p>
 
      <!-- reward -->
      
      <div id="reword-out">
        <div id="reward-btn">
          打赏
        </div>
      </div>
      
    </div>
    

    <!-- copyright -->
    
    <div class="declare">
      <ul class="post-copyright">
        <li>
          <i class="ri-copyright-line"></i>
          <strong>版权声明： </strong>
          
          本博客所有文章除特别声明外，著作权归作者所有。转载请注明出处！
          
        </li>
      </ul>
    </div>
    
    <footer class="article-footer">
       
<div class="share-btn">
      <span class="share-sns share-outer">
        <i class="ri-share-forward-line"></i>
        分享
      </span>
      <div class="share-wrap">
        <i class="arrow"></i>
        <div class="share-icons">
          
          <a class="weibo share-sns" href="javascript:;" data-type="weibo">
            <i class="ri-weibo-fill"></i>
          </a>
          <a class="weixin share-sns wxFab" href="javascript:;" data-type="weixin">
            <i class="ri-wechat-fill"></i>
          </a>
          <a class="qq share-sns" href="javascript:;" data-type="qq">
            <i class="ri-qq-fill"></i>
          </a>
          <a class="douban share-sns" href="javascript:;" data-type="douban">
            <i class="ri-douban-line"></i>
          </a>
          <!-- <a class="qzone share-sns" href="javascript:;" data-type="qzone">
            <i class="icon icon-qzone"></i>
          </a> -->
          
          <a class="facebook share-sns" href="javascript:;" data-type="facebook">
            <i class="ri-facebook-circle-fill"></i>
          </a>
          <a class="twitter share-sns" href="javascript:;" data-type="twitter">
            <i class="ri-twitter-fill"></i>
          </a>
          <a class="google share-sns" href="javascript:;" data-type="google">
            <i class="ri-google-fill"></i>
          </a>
        </div>
      </div>
</div>

<div class="wx-share-modal">
    <a class="modal-close" href="javascript:;"><i class="ri-close-circle-line"></i></a>
    <p>扫一扫，分享到微信</p>
    <div class="wx-qrcode">
      <img src="//api.qrserver.com/v1/create-qr-code/?size=150x150&data=http://example.com/2020/11/11/k8s/K8S%E4%BD%BF%E7%94%A8ceph-csi%E6%8C%81%E4%B9%85%E5%8C%96%E5%AD%98%E5%82%A8%E4%B9%8BCephFS/" alt="微信分享二维码">
    </div>
</div>

<div id="share-mask"></div>  
  <ul class="article-tag-list" itemprop="keywords"><li class="article-tag-list-item"><a class="article-tag-list-link" href="/tags/k8s/" rel="tag">k8s</a></li></ul>

    </footer>
  </div>

   
  <nav class="article-nav">
    
      <a href="/2020/11/11/k8s/Jenkis%20pipeline%E7%BC%96%E5%86%99podTemplate%E5%AE%9E%E7%8E%B0%E5%92%8Ck8s%E5%AF%B9%E6%8E%A5/" class="article-nav-link">
        <strong class="article-nav-caption">上一篇</strong>
        <div class="article-nav-title">
          
            Jenkis pipeline编写podTemplate实现和k8s对接.md
          
        </div>
      </a>
    
    
      <a href="/2020/11/11/k8s/Kubernetes%20%E6%96%B0%E7%8E%A9%E6%B3%95%EF%BC%9A%E5%9C%A8%20yaml%20%E4%B8%AD%E7%BC%96%E7%A8%8B/" class="article-nav-link">
        <strong class="article-nav-caption">下一篇</strong>
        <div class="article-nav-title">Kubernetes 新玩法：在 yaml 中编程.md</div>
      </a>
    
  </nav>

   
<!-- valine评论 -->
<div id="vcomments-box">
  <div id="vcomments"></div>
</div>
<script src="//cdn1.lncld.net/static/js/3.0.4/av-min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/valine@1.4.14/dist/Valine.min.js"></script>
<script>
  new Valine({
    el: "#vcomments",
    app_id: "",
    app_key: "",
    path: window.location.pathname,
    avatar: "monsterid",
    placeholder: "给我的文章加点评论吧~",
    recordIP: true,
  });
  const infoEle = document.querySelector("#vcomments .info");
  if (infoEle && infoEle.childNodes && infoEle.childNodes.length > 0) {
    infoEle.childNodes.forEach(function (item) {
      item.parentNode.removeChild(item);
    });
  }
</script>
<style>
  #vcomments-box {
    padding: 5px 30px;
  }

  @media screen and (max-width: 800px) {
    #vcomments-box {
      padding: 5px 0px;
    }
  }

  #vcomments-box #vcomments {
    background-color: #fff;
  }

  .v .vlist .vcard .vh {
    padding-right: 20px;
  }

  .v .vlist .vcard {
    padding-left: 10px;
  }
</style>

 
     
</article>

</section>
      <footer class="footer">
  <div class="outer">
    <ul>
      <li>
        Copyrights &copy;
        2015-2020
        <i class="ri-heart-fill heart_icon"></i> TzWind
      </li>
    </ul>
    <ul>
      <li>
        
        
        
        由 <a href="https://hexo.io" target="_blank">Hexo</a> 强力驱动
        <span class="division">|</span>
        主题 - <a href="https://github.com/Shen-Yu/hexo-theme-ayer" target="_blank">Ayer</a>
        
      </li>
    </ul>
    <ul>
      <li>
        
        
        <span>
  <span><i class="ri-user-3-fill"></i>访问人数:<span id="busuanzi_value_site_uv"></span></s>
  <span class="division">|</span>
  <span><i class="ri-eye-fill"></i>浏览次数:<span id="busuanzi_value_page_pv"></span></span>
</span>
        
      </li>
    </ul>
    <ul>
      
    </ul>
    <ul>
      
    </ul>
    <ul>
      <li>
        <!-- cnzz统计 -->
        
        <script type="text/javascript" src='https://s9.cnzz.com/z_stat.php?id=1278069914&amp;web_id=1278069914'></script>
        
      </li>
    </ul>
  </div>
</footer>
      <div class="float_btns">
        <div class="totop" id="totop">
  <i class="ri-arrow-up-line"></i>
</div>

<div class="todark" id="todark">
  <i class="ri-moon-line"></i>
</div>

      </div>
    </main>
    <aside class="sidebar on">
      <button class="navbar-toggle"></button>
<nav class="navbar">
  
  <div class="logo">
    <a href="/"><img src="/images/ayer-side.svg" alt="Hexo"></a>
  </div>
  
  <ul class="nav nav-main">
    
    <li class="nav-item">
      <a class="nav-item-link" href="/">主页</a>
    </li>
    
    <li class="nav-item">
      <a class="nav-item-link" href="/archives">归档</a>
    </li>
    
    <li class="nav-item">
      <a class="nav-item-link" href="/categories">分类</a>
    </li>
    
    <li class="nav-item">
      <a class="nav-item-link" href="/tags">标签</a>
    </li>
    
    <li class="nav-item">
      <a class="nav-item-link" target="_blank" rel="noopener" href="http://www.baidu.com">百度</a>
    </li>
    
    <li class="nav-item">
      <a class="nav-item-link" href="/friends">友链</a>
    </li>
    
    <li class="nav-item">
      <a class="nav-item-link" href="/2019/about">关于我</a>
    </li>
    
  </ul>
</nav>
<nav class="navbar navbar-bottom">
  <ul class="nav">
    <li class="nav-item">
      
      <a class="nav-item-link nav-item-search"  title="搜索">
        <i class="ri-search-line"></i>
      </a>
      
      
      <a class="nav-item-link" target="_blank" href="/atom.xml" title="RSS Feed">
        <i class="ri-rss-line"></i>
      </a>
      
    </li>
  </ul>
</nav>
<div class="search-form-wrap">
  <div class="local-search local-search-plugin">
  <input type="search" id="local-search-input" class="local-search-input" placeholder="Search...">
  <div id="local-search-result" class="local-search-result"></div>
</div>
</div>
    </aside>
    <script>
      if (window.matchMedia("(max-width: 768px)").matches) {
        document.querySelector('.content').classList.remove('on');
        document.querySelector('.sidebar').classList.remove('on');
      }
    </script>
    <div id="mask"></div>

<!-- #reward -->
<div id="reward">
  <span class="close"><i class="ri-close-line"></i></span>
  <p class="reward-p"><i class="ri-cup-line"></i>请我喝杯咖啡吧~</p>
  <div class="reward-box">
    
    
  </div>
</div>
    
<script src="/js/jquery-2.0.3.min.js"></script>


<script src="/js/lazyload.min.js"></script>

<!-- Tocbot -->


<script src="/js/tocbot.min.js"></script>

<script>
  tocbot.init({
    tocSelector: '.tocbot',
    contentSelector: '.article-entry',
    headingSelector: 'h1, h2, h3, h4, h5, h6',
    hasInnerContainers: true,
    scrollSmooth: true,
    scrollContainer: 'main',
    positionFixedSelector: '.tocbot',
    positionFixedClass: 'is-position-fixed',
    fixedSidebarOffset: 'auto'
  });
</script>

<script src="https://cdn.jsdelivr.net/npm/jquery-modal@0.9.2/jquery.modal.min.js"></script>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/jquery-modal@0.9.2/jquery.modal.min.css">
<script src="https://cdn.jsdelivr.net/npm/justifiedGallery@3.7.0/dist/js/jquery.justifiedGallery.min.js"></script>

<script src="/dist/main.js"></script>

<!-- ImageViewer -->

<!-- Root element of PhotoSwipe. Must have class pswp. -->
<div class="pswp" tabindex="-1" role="dialog" aria-hidden="true">

    <!-- Background of PhotoSwipe. 
         It's a separate element as animating opacity is faster than rgba(). -->
    <div class="pswp__bg"></div>

    <!-- Slides wrapper with overflow:hidden. -->
    <div class="pswp__scroll-wrap">

        <!-- Container that holds slides. 
            PhotoSwipe keeps only 3 of them in the DOM to save memory.
            Don't modify these 3 pswp__item elements, data is added later on. -->
        <div class="pswp__container">
            <div class="pswp__item"></div>
            <div class="pswp__item"></div>
            <div class="pswp__item"></div>
        </div>

        <!-- Default (PhotoSwipeUI_Default) interface on top of sliding area. Can be changed. -->
        <div class="pswp__ui pswp__ui--hidden">

            <div class="pswp__top-bar">

                <!--  Controls are self-explanatory. Order can be changed. -->

                <div class="pswp__counter"></div>

                <button class="pswp__button pswp__button--close" title="Close (Esc)"></button>

                <button class="pswp__button pswp__button--share" style="display:none" title="Share"></button>

                <button class="pswp__button pswp__button--fs" title="Toggle fullscreen"></button>

                <button class="pswp__button pswp__button--zoom" title="Zoom in/out"></button>

                <!-- Preloader demo http://codepen.io/dimsemenov/pen/yyBWoR -->
                <!-- element will get class pswp__preloader--active when preloader is running -->
                <div class="pswp__preloader">
                    <div class="pswp__preloader__icn">
                        <div class="pswp__preloader__cut">
                            <div class="pswp__preloader__donut"></div>
                        </div>
                    </div>
                </div>
            </div>

            <div class="pswp__share-modal pswp__share-modal--hidden pswp__single-tap">
                <div class="pswp__share-tooltip"></div>
            </div>

            <button class="pswp__button pswp__button--arrow--left" title="Previous (arrow left)">
            </button>

            <button class="pswp__button pswp__button--arrow--right" title="Next (arrow right)">
            </button>

            <div class="pswp__caption">
                <div class="pswp__caption__center"></div>
            </div>

        </div>

    </div>

</div>

<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/photoswipe@4.1.3/dist/photoswipe.min.css">
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/photoswipe@4.1.3/dist/default-skin/default-skin.min.css">
<script src="https://cdn.jsdelivr.net/npm/photoswipe@4.1.3/dist/photoswipe.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/photoswipe@4.1.3/dist/photoswipe-ui-default.min.js"></script>

<script>
    function viewer_init() {
        let pswpElement = document.querySelectorAll('.pswp')[0];
        let $imgArr = document.querySelectorAll(('.article-entry img:not(.reward-img)'))

        $imgArr.forEach(($em, i) => {
            $em.onclick = () => {
                // slider展开状态
                // todo: 这样不好，后面改成状态
                if (document.querySelector('.left-col.show')) return
                let items = []
                $imgArr.forEach(($em2, i2) => {
                    let img = $em2.getAttribute('data-idx', i2)
                    let src = $em2.getAttribute('data-target') || $em2.getAttribute('src')
                    let title = $em2.getAttribute('alt')
                    // 获得原图尺寸
                    const image = new Image()
                    image.src = src
                    items.push({
                        src: src,
                        w: image.width || $em2.width,
                        h: image.height || $em2.height,
                        title: title
                    })
                })
                var gallery = new PhotoSwipe(pswpElement, PhotoSwipeUI_Default, items, {
                    index: parseInt(i)
                });
                gallery.init()
            }
        })
    }
    viewer_init()
</script>

<!-- MathJax -->

<!-- Katex -->

<!-- busuanzi  -->


<script src="/js/busuanzi-2.3.pure.min.js"></script>


<!-- ClickLove -->

<!-- ClickBoom1 -->

<!-- ClickBoom2 -->

<!-- CodeCopy -->


<link rel="stylesheet" href="/css/clipboard.css">

<script src="https://cdn.jsdelivr.net/npm/clipboard@2/dist/clipboard.min.js"></script>
<script>
  function wait(callback, seconds) {
    var timelag = null;
    timelag = window.setTimeout(callback, seconds);
  }
  !function (e, t, a) {
    var initCopyCode = function(){
      var copyHtml = '';
      copyHtml += '<button class="btn-copy" data-clipboard-snippet="">';
      copyHtml += '<i class="ri-file-copy-2-line"></i><span>COPY</span>';
      copyHtml += '</button>';
      $(".highlight .code pre").before(copyHtml);
      $(".article pre code").before(copyHtml);
      var clipboard = new ClipboardJS('.btn-copy', {
        target: function(trigger) {
          return trigger.nextElementSibling;
        }
      });
      clipboard.on('success', function(e) {
        let $btn = $(e.trigger);
        $btn.addClass('copied');
        let $icon = $($btn.find('i'));
        $icon.removeClass('ri-file-copy-2-line');
        $icon.addClass('ri-checkbox-circle-line');
        let $span = $($btn.find('span'));
        $span[0].innerText = 'COPIED';
        
        wait(function () { // 等待两秒钟后恢复
          $icon.removeClass('ri-checkbox-circle-line');
          $icon.addClass('ri-file-copy-2-line');
          $span[0].innerText = 'COPY';
        }, 2000);
      });
      clipboard.on('error', function(e) {
        e.clearSelection();
        let $btn = $(e.trigger);
        $btn.addClass('copy-failed');
        let $icon = $($btn.find('i'));
        $icon.removeClass('ri-file-copy-2-line');
        $icon.addClass('ri-time-line');
        let $span = $($btn.find('span'));
        $span[0].innerText = 'COPY FAILED';
        
        wait(function () { // 等待两秒钟后恢复
          $icon.removeClass('ri-time-line');
          $icon.addClass('ri-file-copy-2-line');
          $span[0].innerText = 'COPY';
        }, 2000);
      });
    }
    initCopyCode();
  }(window, document);
</script>


<!-- CanvasBackground -->


    
  </div>
</body>

</html>